In [1]:
%matplotlib inline
import matplotlib.pyplot as plt
import pandas as pd
pd.set_option('display.max_columns', 500)
import seaborn as sns
sns.set_style("dark")
plt.rcParams['figure.figsize'] = 8, 6
from sklearn.cross_validation import StratifiedKFold, KFold
import pickle
import time
import numpy as np
import lasagne
from lasagne.layers import InputLayer, DenseLayer, NonlinearityLayer
from lasagne.layers.dnn import Conv2DDNNLayer as ConvLayer
from lasagne.layers import Pool2DLayer as PoolLayer
from lasagne.layers import DropoutLayer
from sklearn import metrics
from lasagne.nonlinearities import softmax
from lasagne.utils import floatX
import matplotlib.pyplot as plt
import io
import skimage.transform
import optparse
import os
import theano.tensor as T
import theano
import json
import glob
Using gpu device 0: Tesla K40c (CNMeM is disabled, cuDNN 4007)
In [2]:
base_path = '/home/pnesterov/babak/deep_fashion/experiments/classifiers/shoes8classification/exp3/part1/'

def build_vgg16():
    net = {}
    net['input'] = InputLayer((None, 3, 224, 224))
    net['conv1_1'] = ConvLayer(
        net['input'], 64, 3, pad=1, flip_filters=False)
    net['conv1_2'] = ConvLayer(
        net['conv1_1'], 64, 3, pad=1, flip_filters=False)
    net['pool1'] = PoolLayer(net['conv1_2'], 2)
    net['conv2_1'] = ConvLayer(
        net['pool1'], 128, 3, pad=1, flip_filters=False)
    net['conv2_2'] = ConvLayer(
        net['conv2_1'], 128, 3, pad=1, flip_filters=False)
    net['pool2'] = PoolLayer(net['conv2_2'], 2)
    net['conv3_1'] = ConvLayer(
        net['pool2'], 256, 3, pad=1, flip_filters=False)
    net['conv3_2'] = ConvLayer(
        net['conv3_1'], 256, 3, pad=1, flip_filters=False)
    net['conv3_3'] = ConvLayer(
        net['conv3_2'], 256, 3, pad=1, flip_filters=False)
    net['pool3'] = PoolLayer(net['conv3_3'], 2)
    net['conv4_1'] = ConvLayer(
        net['pool3'], 512, 3, pad=1, flip_filters=False)
    net['conv4_2'] = ConvLayer(
        net['conv4_1'], 512, 3, pad=1, flip_filters=False)
    net['conv4_3'] = ConvLayer(
        net['conv4_2'], 512, 3, pad=1, flip_filters=False)
    net['pool4'] = PoolLayer(net['conv4_3'], 2)
    net['conv5_1'] = ConvLayer(
        net['pool4'], 512, 3, pad=1, flip_filters=False)
    net['conv5_2'] = ConvLayer(
        net['conv5_1'], 512, 3, pad=1, flip_filters=False)
    net['conv5_3'] = ConvLayer(
        net['conv5_2'], 512, 3, pad=1, flip_filters=False)
    net['pool5'] = PoolLayer(net['conv5_3'], 2)
    net['fc6'] = DenseLayer(net['pool5'], num_units=4096)
    net['fc6_dropout'] = DropoutLayer(net['fc6'], p=0.5)
    net['fc7'] = DenseLayer(net['fc6_dropout'], num_units=4096)
    net['fc7_dropout'] = DropoutLayer(net['fc7'], p=0.5)
    net['fc8'] = DenseLayer(
        net['fc7_dropout'], num_units=1000, nonlinearity=None)
    net['prob'] = NonlinearityLayer(net['fc8'], softmax)
    return net

def transfer_vgg16_add_fc1024_do_fc256_do_fc(net, n_output):
    net = lasagne.layers.get_all_layers(net)[-7]
    net = DenseLayer(net, num_units=1024)
    net = DropoutLayer(net, p=0.5)
    net = DenseLayer(net, num_units=256)
    net = DropoutLayer(net, p=0.5)
    net = DenseLayer(net, num_units=n_output, nonlinearity=None)
    net = NonlinearityLayer(net, softmax)
    return net

def transfer_vgg16_addBnFcBnFcBnFc(net, n_output):
    net = lasagne.layers.get_all_layers(net)[-7]
    net = lasagne.layers.BatchNormLayer(net)
    net = DenseLayer(net, num_units=1024, nonlinearity=None)
    net = lasagne.layers.BatchNormLayer(net)
    net = NonlinearityLayer(net, lasagne.nonlinearities.rectify)
    net = DenseLayer(net, num_units=256, nonlinearity=None)
    net = lasagne.layers.BatchNormLayer(net)
    net = NonlinearityLayer(net, lasagne.nonlinearities.rectify)
    net = DenseLayer(net, num_units=n_output, nonlinearity=None)
    net = NonlinearityLayer(net, softmax)
    return net

def vgg2img(img, mean_values_bgr):
    return (np.swapaxes(np.swapaxes(img, 0, 1), 1, 2) + mean_values_bgr)[:, :, ::-1]

def img2vgg(img, mean_values_bgr):
    return np.swapaxes(np.swapaxes(floatX(img[:, :, ::-1] - mean_values_bgr), 1, 2), 0, 1)


def read_image_reduce_min(fname, mean_values_bgr=None, out_size=224,
                          n_filters_min=None, n_filters_max=None, replace_filters=True):
    img = plt.imread(fname, fname.split('.')[-1])
    h, w, _ = img.shape
    if h < w:
        img = skimage.transform.resize(img, (out_size, w*out_size/h), preserve_range=True)
    else:
        img = skimage.transform.resize(img, (h*out_size/w, out_size), preserve_range=True)
    img_raw = np.copy(img).astype('uint8')
    if n_filters_min is not None:
        img = augmentation(img, n_filters_min, n_filters_max, replace_filters)
        h, w, _ = img.shape
        if h < w:
            img = skimage.transform.resize(img, (out_size, w*out_size/h), preserve_range=True)
        else:
            img = skimage.transform.resize(img, (h*out_size/w, out_size), preserve_range=True)
    h, w, _ = img.shape
    img = img[h//2 - int(np.floor(out_size/2.0)):h//2 + int(np.ceil(out_size/2.0)),
              w//2 - int(np.floor(out_size/2.0)):w//2 + int(np.ceil(out_size/2.0))]
    if mean_values_bgr is not None:
        img = img2vgg(img, mean_values_bgr)
    return img_raw, img


def read_image_reduce_max(fname, mean_values_bgr=None, out_size=224,
                          n_filters_min=None, n_filters_max=None, replace_filters=True):
    img = plt.imread(fname, fname.split('.')[-1])
    img_raw = np.copy(img).astype('uint8')
    m = np.zeros((out_size, out_size, 3)) + 255
    h, w, _ = img.shape
    if n_filters_min is not None:
        if h < w:
            img = skimage.transform.resize(img, (h*out_size/w, out_size), preserve_range=True)
        else:
            img = skimage.transform.resize(img, (out_size, w*out_size/h), preserve_range=True)
        img = augmentation(img, n_filters_min, n_filters_max, replace_filters)
    h, w, _ = img.shape
    if h < w:
        img = skimage.transform.resize(img, (h*out_size/w, out_size), preserve_range=True)
        x = (out_size - img.shape[0])/2
        m[x:(x + img.shape[0]), :, :] = img
        img = m
    else:
        img = skimage.transform.resize(img, (out_size, w*out_size/h), preserve_range=True)
        y = (out_size - img.shape[1])/2
        m[:, y:(y + img.shape[1]), :] = img
        img = m
    if mean_values_bgr is not None:
        img = img2vgg(img, mean_values_bgr)
    return img_raw, img


def read_image_change_aspect(fname, mean_values_bgr=None, out_size=224,
                             n_filters_min=None, n_filters_max=None, replace_filters=True):
    img = plt.imread(fname, fname.split('.')[-1])
    img_raw = np.copy(img).astype('uint8')
    if n_filters_min is not None:
        img = skimage.transform.resize(img, (out_size, out_size), preserve_range=True)
        img = augmentation(img, n_filters_min, n_filters_max, replace_filters)
    img = skimage.transform.resize(img, (out_size, out_size), preserve_range=True)
    if mean_values_bgr is not None:
        img = img2vgg(img, mean_values_bgr)
    return img_raw, img


image_readers = {
    'reduce_min': read_image_reduce_min,
    'reduce_max': read_image_reduce_max,
    'change_aspect': read_image_change_aspect
}

read_image = image_readers['reduce_max']

vgg_19_mean_values = pickle.load(open('/home/pnesterov/babak/deep_fashion/models/vgg16.pkl'))['mean value']
data = json.load(open(os.path.join(base_path, 'train_val_test_split.json'), 'r'))
class_names = map(lambda t: t[0], sorted(data.items(), key=lambda t: t[1]['id']))


net = build_vgg16()['prob']
net = transfer_vgg16_addBnFcBnFcBnFc(net, len(data))
with np.load(os.path.join(base_path, 'trained_net_3.npz')) as f:
    params = [f['arr_%d' % i] for i in range(len(f.files))]
lasagne.layers.set_all_param_values(net, params)


def predict_batch(files, class_names):
    mat = np.array([t[1] for t in map(lambda f: read_image(f, vgg_19_mean_values), files)])
    prob = np.array(lasagne.layers.get_output(net, mat, deterministic=True).eval())
    return map(lambda i: sorted(zip(class_names, prob[i, :]), key=lambda t: t[1], reverse=True), 
               range(prob.shape[0]))
In [3]:
pool = lasagne.layers.get_all_layers(net)[-10]
In [ ]:
 
In [4]:
files = glob.glob('/home/pnesterov/babak/deep_fashion/data/tmp/*.jpg')
In [5]:
for fname in files:
    img_raw, img = read_image(fname, vgg_19_mean_values)
    m = prob = np.array(lasagne.layers.get_output(pool, img[np.newaxis, :], 
                                                  deterministic=True).eval()).squeeze().sum(axis=0)
    m = (m - m.min())/(m.max() - m.min())
    
    plt.figure(figsize=(16, 16), facecolor='w')
    plt.subplot(1, 3, 1)
    plt.axis('off')
    plt.imshow(vgg2img(img, vgg_19_mean_values).astype(np.uint8))
    plt.subplot(1, 3, 2)
    plt.axis('off')
    plt.imshow(m, cmap='hot', interpolation='none')
    plt.subplot(1, 3, 3)
    plt.axis('off')
    plt.imshow(vgg2img(img, vgg_19_mean_values).astype(np.uint8))
    plt.imshow(skimage.transform.resize(m, (img.shape[1], img.shape[2]), preserve_range=True), cmap='hot', alpha=0.7)
    plt.show()
In [ ]:
 
In [ ]:
 
In [ ]:
 
In [ ]:
 
In [ ]:
 
In [ ]:
 
In [ ]:
 
In [ ]:
 
In [ ]: